package com.whu.lsj;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

/**
 * 原始语料的部分处理工作
 * Created by Enginner-Jie on 2016/8/27.
 */
public class DealText {

	public static void main(String args[]) throws IOException {
		new DealText().TrainToTest("data/cs_train_part.utf8");
	}
	/**
	 * scan() 扫描所有行，把文本读入List
	 */
	public List<String> scan(String path){
		File test_file = new File(path);

		List<String> sentences = new ArrayList<>();  //用list存储所有句子
		BufferedReader br = null;
		try {
			br = new BufferedReader(new FileReader(test_file));
			String line;
			while ((line = br.readLine()) != null) {
				sentences.add(line);
			}
		}catch (IOException e){
			e.printStackTrace();
		}finally {
			if (br != null){
				try {
					br.close();
				} catch (IOException e) {
					e.printStackTrace();
				}
			}
		}
		return  sentences;
	}

	/**
	 * removeSame() 去掉相同的句子
	 * 获取原始句子，去掉term标签
	 * @param path
	 */
	public void removeSame(String path) {
		File result = new File("F:\\桌面\\term_new\\all_result03");

		BufferedWriter bw = null;
		try {
			bw = new BufferedWriter(new FileWriter(result));
			List<String> sentences = scan(path);

			List<Integer> lineNum = new ArrayList<>();
			//保留最后一行
			for (int i = sentences.size()-1 ;i >= 0;i--){
				String oneline = sentences.get(i);
				String ss[] = oneline.split("\t",0);
				int linenum = Integer.parseInt(ss[0]);
				if (!lineNum.contains(linenum)){
					lineNum.add(linenum);
					bw.write(oneline +"\n");
				}else{
					sentences.remove(i);
				}
			}
			System.out.println("Finished!");
		} catch (IOException e){
			e.printStackTrace();
		}finally {
			if (bw != null){
				try {
					bw.close();
				} catch (IOException e) {
					e.printStackTrace();
				}
			}
		}
	}

	/**
	 * 读写文件,拿掉term,训练转测试
	 * @param path
	 */
	public void TrainToTest(String path) throws IOException {
		long start = System.currentTimeMillis();
		File test_file = new File(path);
		File crf_test_file = new File("data/cs_test_part.utf8");


		BufferedReader br = null;
		BufferedWriter bw = null;
		try {
			br = new BufferedReader(new FileReader(test_file));
			bw = new BufferedWriter(new FileWriter(crf_test_file));
			String line;
			while ((line = br.readLine()) != null) {
				bw.write(getSentence(line)+"\n");
			}
		} catch (IOException e){
			e.printStackTrace();
		}finally {
			bw.close();
			br.close();
		}
	}

	/**
	 * 获取原始句子，去掉term标签
	 * @param line
	 */
	public static String getSentence(String line) {

		StringBuffer sb = new StringBuffer(line);

		int t1begin = line.indexOf("<term id=\"");
		int t1end = line.indexOf("\">");

		String part1 = new String(sb.delete(t1begin,t1end + 2));
		String pureSentence = part1.replace("</term>","");
		//	System.out.println(pureSentence);
		return pureSentence;
	}
}


